package data.process;

import static common.Common.docFolder;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.Map.Entry;
import java.util.regex.Pattern;

import javax.swing.JProgressBar;

import common.Common;

import data.news.Article;
import data.news.ArticleIterator;
import data.news.Keyword;
import data.news.KeywordIterator;
import data.news.db.NewsDB;
import prefuse.data.Schema;
import prefuse.data.Table;
import prefuse.data.Tuple;
import weeny.pre.io.XmlTableUtil;
import weeny.recycle.Control;
import weeny.util.Counter;
import weeny.util.logs.Debug;

@SuppressWarnings("deprecation")
public class Collector {
	public static int TOPXXX = 40;
	public static int MAX_CNT = 1000;
	public static Map<String, String> mapper = new HashMap<String, String>();
	static{
		mapper.put("Merck & &", "Merck");
		mapper.put("Merck KGaA", "Merck");
		mapper.put("Biomira Biomira", "Biomira");
		mapper.put("Biomira Inc", "Biomira");
		mapper.put("Arena Pharmaceuticals,", "Arena");
		mapper.put("M2 Communications Ltd", "M2 Communications");
		mapper.put("Nastech Pharmaceutical Company Company", "Nastech");
		mapper.put("VIOXX", "Vioxx");
		mapper.put("SANCTURA", "Sanctura");
		mapper.put("ESTRASORB", "Estrasorb");
	}
	
	public static Schema COUNTER_SCHEMA = new Schema();
	public static Schema DOC_SCHEMA = new Schema();
	static{
		COUNTER_SCHEMA.addColumn("value", String.class);
		COUNTER_SCHEMA.addColumn("count", int.class);
		COUNTER_SCHEMA.addColumn("type",  String.class);

		DOC_SCHEMA.addColumn(NewsDB.FLD_ID,       long.class);
		DOC_SCHEMA.addColumn(NewsDB.FLD_TITLE,    String.class);
		DOC_SCHEMA.addColumn(NewsDB.FLD_DATE,     int.class);
		DOC_SCHEMA.addColumn(NewsDB.FLD_ART_TYPE, String.class);
		DOC_SCHEMA.addColumn(NewsDB.FLD_PRE_TYPE, String.class);
		DOC_SCHEMA.addColumn(NewsDB.FLD_SYN_DICT, String.class);
		DOC_SCHEMA.addColumn(NewsDB.FLD_SRC_NAME, String.class);
		DOC_SCHEMA.addColumn("length",   int.class);
	}
	public static List<Long> ids(String companyName){
		String docFolder = docFolder(companyName);
		List<String> files = Control.searchFilesWithExt("stas", new File(docFolder));
		List<Long> ids = new ArrayList<Long>();
		for (String fileName : files){
			long id = Common.idInFileName(fileName);
			ids.add(id);
		}
		return ids;
	}
	
	public static Table topCountOfCategory(List<Long> ids, String category, int topCnt, JProgressBar bar){
		String upperCaseValue = category.toUpperCase();
		Map<String, String> low2Key = new TreeMap<String, String>();
		low2Key.clear();
		Counter<String> count = new Counter<String>();
		int cnt = 0;
		if (bar != null) {
			bar.setMaximum(ids.size()+1);
		}
		for (Long id : ids) {
			Article article = NewsDB.article(id);
			KeywordIterator it = article.keywords("pn");
			while (it.hasNext()){
				Keyword key = it.next();
				if (upperCaseValue.equals(key.getAttrValue("category"))) {
					String lowerContent = Collector.safe(key.getWord());
					if (lowerContent == null) {
						continue;
					}
					count.increase(lowerContent);
					if (low2Key.containsKey(lowerContent)){
						if (key.getWord().length()<low2Key.get(lowerContent).length()) {
							low2Key.put(lowerContent, key.getWord());
						}
					}
					else{
						low2Key.put(lowerContent, key.getWord());
					}
				}
			}
			cnt++;
			if (bar != null)
				bar.setValue(cnt);
			if (cnt%10 == 1) {
				System.out.println("reading file #"+cnt);
			}
		}
		List<Entry<String, Number>> list = count.topByCount(topCnt);
		Table table = COUNTER_SCHEMA.instantiate();
		for(Iterator<Entry<String, Number>> iter = list.iterator(); iter.hasNext(); ){
			Entry<String, Number> word = iter.next();
			if (word.getValue().intValue() >=2){
				int row = table.addRow();
				table.setString(row, "value", low2Key.get(word.getKey()));
				table.setInt(row, "count", word.getValue().intValue());
				table.setString(row, "type", category);
			}
		}
		return table;
	}
	
	/**
	 * output position and sentiment in every document
	 * word1 [a,b,c]
	 * word1_sent [a,b,c]
	 * ...
	 * id XXXX
	 */
	public static void outputSentsMarks(List<Long> ids, String output, Set<String> set, JProgressBar bar){
		Map<String, String> low2Key = new TreeMap<String, String>();
		for (String string : set) {
			if(mapper.containsKey(string)) {
				string = mapper.get(string);
			}
			low2Key.put(safe(string), trim(string));
		}
		Table nodeTable = new Table();
		for (String string : low2Key.values()) {
			nodeTable.addColumn(string, ArrayList.class, null);
			nodeTable.addColumn(string+"_sent", ArrayList.class, null);
		}
		nodeTable.addColumn("id", long.class);

		Map<String, List<Integer>> tempMap = new TreeMap<String, List<Integer>>();
		for (String string : low2Key.values()) {
			tempMap.put(string, new ArrayList<Integer>());
			tempMap.put(string+"_sent", new ArrayList<Integer>());
		}

		int cnt = 0;
		if (bar != null) {
			bar.setMaximum(ids.size()+1);
		}
		for(Long id : ids){
			Article art = NewsDB.article(id);
			cnt ++;
			if(bar != null)
				bar.setValue(cnt);
			if (cnt % 100 == 0) {
				System.out.println("processing file #"+cnt);
			}
			for (String string : tempMap.keySet()) {
				if (!tempMap.get(string).isEmpty()) {
					tempMap.put(string, new ArrayList<Integer>());
				}
			}

			Tuple artTuple = nodeTable.getTuple(nodeTable.addRow()); 
			artTuple.setLong("id", art.getArticleID());
			BufferedReader br = new BufferedReader(new StringReader(art.getContent()));
			int lineCnt = 0;
			String line = null;
			try {
				while((line = br.readLine())!=null){
					if (line.length()<8) {
						continue;
					}
					KeywordIterator iter = new KeywordIterator(line, "pn");
					while (iter.hasNext()){
						Keyword key = iter.next();
						String safeValue = Collector.safe(key.getWord());
						if (Collector.mapper.containsKey(key.getWord())) {
							safeValue = Collector.safe(Collector.mapper.get(key.getWord()));
						}
						if (safeValue == null) {
							continue;
						}
						if (low2Key.keySet().contains(safeValue)) {
							tempMap.get(low2Key.get(safeValue)).add(lineCnt);
							int sent = 0;
							////just use GENERAL attributes, it is not useful
							//String g0 = key.getAttrValue("GENERAL0");
							//if (g0 != null) {
							//		sent += Integer.parseInt(g0);
							//}
							//String g1 = key.getAttrValue("GENERAL1");
							//if (g1 != null) {
							//		sent += Integer.parseInt(g1);
							//}
							///////////////////////////////////////////////
							sent = 0;
							for(int i = 0; i<sentiGroups.length; i++){
								for(int j = 0; j<=1; j++){
									String g = key.getAttrValue(sentiGroups[i]+j);
									if (g != null) {
										sent += Integer.parseInt(g);
									}
								}
							}
							tempMap.get(low2Key.get(safeValue)+"_sent").add(sent);
						}
					}
					lineCnt++;
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
			for(String string : tempMap.keySet()){
				if (!tempMap.get(string).isEmpty()) {
					artTuple.set(string, tempMap.get(string));
				}
			}
		}
		XmlTableUtil.writeTable(nodeTable, output);
	}

	public static String[] sentiGroups = new String[]{
		"GENERAL", "BUSINESS", "CRIME", "HEALTH", "POLITICS", "SPORTS","MEDIA"
	};
	
	public static Set<String> topKeywords(String companyName, String categoryValue, int count){
		Set<String> result = new HashSet<String>();
		String cachedFile = Common.wheelFolder(companyName)+categoryValue.toUpperCase()+MAX_CNT+".xml";
		if (!new File(cachedFile).exists()) {
			List<Long> ids = ids(companyName);
			Table table =  topCountOfCategory(ids, categoryValue, MAX_CNT, null);
			XmlTableUtil.writeTable(table, cachedFile);
		}
		else{
			Debug.msg("read from cached file: "+cachedFile);
		}
		Table cachedTable = XmlTableUtil.readTable(cachedFile);
		Set<String> temp = new HashSet<String>();
		for(Iterator<Tuple> it = cachedTable.tuples(); it.hasNext();){
			Tuple tp = it.next();
			String key = tp.getString("value");
			if (Collector.trimSafe(key) == null) {
				continue;
			}
			if (Collector.mapper.containsKey(key)) {
				temp.add(Collector.mapper.get(key));
				result.add(key);
			}
			else{
				temp.add(key);
				result.add(key);
			}
			if (temp.size() == count) {
				System.out.println(temp);
				return result;
			}
		}
		return result;
	}
	
	public static void outputDocInfos(BufferedReader br, String fileName){
		ArticleIterator it = new ArticleIterator(br);
		Table table = DOC_SCHEMA.instantiate();
		int cnt =0;
		while(it.hasNext()){
			cnt++;
			if (cnt%100 == 0) {
				System.out.println("processing doc #"+cnt);
			}
			Article art = it.next();
			Tuple tuple = table.getTuple(table.addRow());
			tuple.setLong  (NewsDB.FLD_ID,       art.getArticleID());
			tuple.setString(NewsDB.FLD_TITLE,    art.getTitle());
			tuple.setInt   (NewsDB.FLD_DATE,     art.getDate());
			tuple.setInt   ("length",            art.getSentenceCount());
			tuple.setString(NewsDB.FLD_ART_TYPE, art.getArticletype());
			tuple.setString(NewsDB.FLD_PRE_TYPE, art.getPredictedarticletype());
			tuple.setString(NewsDB.FLD_SYN_DICT, art.getSyndicate());
			tuple.setString(NewsDB.FLD_SRC_NAME, art.getSourcename());
		}
		XmlTableUtil.writeTable(table, fileName);
	}

	static String safe(String str){
		if (str.indexOf(' ')<0) {
			return str;
		}
		String t = Collector.trimSafe(str);
		while(true){
			String r = Collector.trimSafe(t);
			if (t == null || t.equals(r)) {
				break;
			}
			t = r;
		}
		return t;
	}

	static String trimSafe(String str){
		if (str == null) {
			return str;
		}
		str = str.toLowerCase();
		str = str.replaceAll("\\p{Punct}", "");
		str = str.replaceAll(" +", " ");
		str = str.trim();
		if (str.endsWith(" ltd")) {
			str = str.substring(0, str.length()-3);
		}
		if (str.endsWith("inc")) {
			str = str.substring(0, str.length()-3);
		}
		if (str.endsWith(" co")) {
			str = str.substring(0, str.length()-2);
		}
		if (str.endsWith("corp")) {
			str = str.substring(0, str.length()-4);
		}
		str = str.trim();
		if (str.contains("target") || str.length()<=1) {
			return null;
		}
		if (str.equals("ltd")) {
			return null;
		}
		if (str.equals("co")) {
			return null;
		}
		return str;
	}

	public static Pattern pat = Pattern.compile(" ,");
	public static String trim(String str){
		return pat.matcher(str).replaceAll("");
	}
}
